#!/usr/bin/perl
#data_prepare.pl
#prepares data for the LAM Aligner
#USAGE: perl data_prepare.pl -sent SENTENCEFILE -list FILELIST -conv CONVERTTABLES -wdir WAVEDIR [-slist SENTENCELIST -wlist WORDLIST -h]
#no extra lines; no umlauts; special characters like "&", "§" need to be written out (und, paragraph); digits need to be written out;
#the number of transcribed units HAS TO equal the number of orthographic units
#the input format for the SENTENCEFILE: transcription line, orthographic line, empty line and s.o.; the first Word" in each transcription line has to be a 
#sentence identifier or a non-significant string
#the FILELIST has to contain all audio data filenames with or without extension
#CONVERTTABLES contain a list with two columns; the first column contains the units to convert (e.g. transcription); the second one the convertion code. 
#WAVEDIR contains the corpus wav-Files
#SENTENCELIST: prints out a list of all sentences in one file (optional)
#WORDLIST: prints out all words (orthographical) in one file (optional)
#!Attention: the code characters may NOT appear in the units to convert (e.g. binary code is usable, if the transcription does not contain 0s or 1s)
#!Atention: all quantifiers [?*+] and special characters for regular expressions (such as "\" or "/") in transcription in the SENTENCEFILE and CONVERTTABLES (or UNITLIST if using code.pl script) have to be substituted (e.g. "?" with "!")

#Version: 1.1
#@autor: Igor Jauk
#Changelist:
#config class support
#flexible wavdir (old "data" directory, where the txt-Files are being written to)
#error message if found a different transcription of the same word

use FindBin qw($Bin);
use lib "$Bin/../libs";
use config;

my @dict;
my @dictstr;
my @convert;
my @wlist;
my $clength;
my $USAGE = "perl data_prepare.pl -sent SENTENCEFILE -list FILELIST -conv CONVERTTABLES -wdir WAVEDIR [-slist SENTENCELIST -wlist WORDLIST -h]";

main();

sub main {
	my $cl = config->new(\@ARGV);
	if ($cl->optionflag("-h")) {
		print $USAGE."\n";
		exit;
	}

	die "error: no WAVDIR specified\nusage: $USAGE\n" if (!$cl->optionflag("-wdir"));
	die "error: no sentence file specified\nusage: $USAGE\n" if (!$cl->optionflag("-sent"));

	my $wdir = $cl->optionarg("-wdir"); 

	my $file = $cl->optionarg("-sent");
	open IN, "$file" or die "could not open file $file\n";
	chomp(my @cont = <IN>);
	close(IN);

	my $listfile = $cl->optionarg("-list");
	open LIST, "$listfile" or die "could not open filelist $list\n";
	chomp(my @list = <LIST>);
	close(LIST);

	my $convertfile = $cl->optionarg("-conv");
	open CONV, "$convertfile" or die "could not open convert tables\n";
	chomp(@convert = <CONV>);
	close(CONV);

	my $sl = 0;
	if ($cl->optionflag("-slist")) {
		my $slist = $cl->optionarg("-slist");
		open SLIST, ">$slist" or die "could not create sentence list file $slist\n";
		$sl = 1;
	}
	$clength = @convert;
	
	my $listlength = @list;
	for (my $i = 0; $i < $listlength; $i++) {
		$list[$i] =~ s/\..*$//;
	}

	my $contlength = @cont;
	my $n = 0;
	#printing out sentences in orthographic form to files corresponding to the audio data files (extended from the FILELIST)
	for(my $i = 0; $i < $contlength; $i++) {
		if ($cont[$i] eq "" || !defined $cont[$i]) {
			next;
		}
		$cont[$i+1] =~ s/ü/ue/ig;
		$cont[$i+1] =~ s/ö/oe/ig;
		$cont[$i+1] =~ s/ä/ae/ig;
		$cont[$i+1] =~ s/ß/ss/ig;
		$cont[$i+1] =~ tr/éèúùíìóòáàýỳ/ÉÈÚÙÍÌÓÒÁÀÝỲ/;
		$cont[$i+1] =~ s/[']//ig;
		$cont[$i+1] =~ s/[\/\.!,\\:\?"-;]/ /ig;
		$cont[$i+1] =~ s/&/ und /ig;
		$cont[$i+1] =~ tr/A-Z/a-z/;
		$cont[$i+1] =~ s/ +/ /g;
		$cont[$i] =~ s/ +/ /g;
		open OUT, ">$wdir/$list[$n].txt" or die "could not create file $list[$n].txt\n";
		print OUT $cont[$i+1]."\n";
		close(OUT);
		if ($sl) {
			print SLIST $cont[$i+1]."\n";
		}
		$n++;
		createDict($cont[$i], $cont[$i+1], $cl);
		$i++;
	}
	if ($sl) {
		close(SLIST);
	}
	createDictStr();

	open OUT, ">dict" or die "could not create dictionary file\n";
	my @sorted = sort(@dictstr);			#the entries in the dictionary file HAVE TO be sorted
	foreach (@sorted) {
		print OUT $_."\n";
	}
	close(OUT);

	if ($cl->optionflag("-wlist")) {
		my $dictl = @dict;
		my $wl = @wlist;
		my $wlistfile = $cl->optionarg("-wlist");
		open WLIST, ">$wlistfile" or die "could not create wlist file $wlistfile\n";
		my @sortedwords = sort(@wlist);
		foreach (@sortedwords) {
			$_ =~ tr/a-z/A-Z/;
			print WLIST $_."\n";
		}
		close(WLIST);
	}
	print "done\n";
}

#creates a two-dimensional dictionary array; each word in the dictionary appears only once
sub createDict {
	my $trline = shift;
	my $oline = shift;
	my $cl = shift;
	my @trans = split(/ +/,$trline);
	my @orth = split(/ +/,$oline);
	my $snumber = $trans[0];
	shift(@trans);
	my $trlength = @trans;
	my $strlength = @orth;
	if ($trlength != $strlength) {
		print STDERR "in sentence number $snumber: the amount of orthographic units does not equal the amount of transcripted units\n";
		print STDERR "Transcription: $trline\nOrthographic: $oline\n";
		print STDERR "Words in transcription: $trlength\nWords in the orthographic form: $strlength\n";
		die "aborting\n";
	}
	my $dictl = @dict;
	my $flag = 0;

	for (my $i = 0; $i<$trlength; $i++) {
		for (my $j = 0; $j<$dictl; $j++) {
			if ($orth[$i] eq $dict[$j][0]) {
				$flag = 1;
				if ($trans[$i] ne $dict[$j][1]) {
					die "error: the transcription of the word \"$orth[$i]\" in the sentence $snumber disagrees with the previous stored one (new: /$trans[$i]/ previous: /$dict[$j][1]/)\n";
				}
			}
		}
		if (!$flag) {
			$dict[$dictl][0] = $orth[$i];
			$dict[$dictl][1] = $trans[$i];
			$wlist[$dictl] = $orth[$i];
			$dictl = @dict;
		}
		$flag=0;
	}
	if ($cl->optionflag("-wlist")) {
		my $wl = @wlist;
		for (my $i = 0; $i<$strlength; $i++) {
			for (my $j = 0; $j<$wl; $j++) {
				if ($orth[$i] eq $wlist[$j]) {
					$flag = 1;
				}
			}
			if (!$flag) {
				$wlist[$wl] = $orth[$i];
				$wl = @wlist;
			}
			$flag=0;
		}
	}
}

#creates a one-dimensional dictionary array
sub createDictStr {
	my $dictl = @dict;
	for (my $i = 0; $i < $dictl; $i++) {
		my $crypted = cryptEntry($dict[$i][1]);
		$dict[$i][0] =~ tr/a-z/A-Z/;
		$crypted =~ tr/0/x/;
		$crypted =~ tr/1/y/;
		$dictstr[$i] = "$dict[$i][0] $crypted";
		$dictstr[$i] =~ s/ +/ /g;
	}
}

#codes the dictionary entries based on the CONVERTTABLES
sub cryptEntry {
	my $entry = shift;
	my $buffer = $entry;
	for (my $i=0; $i<$clength; $i++) {
		my @buff = split(/ +/,$convert[$i]);
		if ($entry =~ /$buff[0]/g) {
			#print $entry."\n";
			#print $buff[0]."     ". $entry."\n";
			$entry =~ s/$buff[0]/_$buff[1]#/g;
		}
	}
	$entry =~ s/[_#\\'%\.]+/ /g;
	$entry =~ s/ +/ /g;

	if ($entry =~ /[^01 ]/) {
		print STDERR "unit or part(s) of the unit $buffer not found in convert tables.\n";
		print STDERR "converted unit: $entry\n";
		die "aborting\n";
	}
	
	#if trying to code this way, the second of two consecutive units of the same type is not getting
	# converted (!ERROR); maybe usage of "_ #"context could help (see line 173:  $entry =~ s/[_#\\]+/ /g;
	# uncomment that line and try to substitute those symbols later
	#for (my $i=0; $i<$clength;$i++) {
	#	my @buff = split(/ +/,$convert[$i]);
	#	if ($entry =~ / $buff[1] /g) {
	#		$entry =~ s/ $buff[1] / $buff[2] /g;
	#	}
	#}

	$entry =~ s/ +/ /g;
	$entry =~ s/ +$//;
	return($entry);
}
